//============================================================================
// Name        : Parser.cpp
// Author      : lucmir@dcc.ufmg.br
// Description :
//============================================================================

#include "Parser.h"


Parser::Parser(std::string inputDirectory, std::string indexFileName) {
	this->reader = new RICPNS::CollectionReader(inputDirectory, indexFileName);
	this->doc.clear();
}

Parser::~Parser() {
	delete this->reader;
}

bool Parser::getNextDocument() {
	return ( this->reader->getNextDocument(this->doc) );
}

void Parser::parseDocument(Run *run, Lexon *lexon, DocumentIndex *docIdx,
	std::ofstream *docLinksFile, AnchorText *anchorText) {

	std::string token;
	char *tk;
	unsigned int termId, docId, refDocId;
	std::string text, link;
	htmlcxx::HTML::ParserDom parser;
	tree<htmlcxx::HTML::Node> dom;
	tree<htmlcxx::HTML::Node>::iterator it, end;
	std::string hrefTag("href");
	std::map<std::string, std::string> mAttributes;
	std::map<std::string, std::string>::iterator mAttributesItr;
	std::list<unsigned int> docLinks;
	std::list<unsigned int>::iterator docLinksItr;
	bool inLinkTag = false;

	// Get URL
	std::string url = this->doc.getURL();

	// Get document id
	docId = docIdx->getDoc(url);
	if(docId == NOT_FOUND) {
		return;
	}

	// Get html
	dom = parser.parseTree(this->doc.getText());
	it = dom.begin();
	end = dom.end();

	for (; it != end; ++it) {

		// Skip scripts
		if(strcmp(it->tagName().c_str(), "script") == 0) {
			it.skip_children();
		}

		// Links
		if (strcmp(it->tagName().c_str(), "a") == 0) {
			it->parseAttributes();
			mAttributes = it->attributes();
			if(!mAttributes.empty()) {
				mAttributesItr = mAttributes.find(hrefTag);
				if(mAttributesItr != mAttributes.end()) {
					link = mAttributesItr->second;
					refDocId = docIdx->getDoc(link);
					if(refDocId != NOT_FOUND && refDocId != docId) {
						docLinks.push_back(refDocId);
					}
				}
			}
		}

		// Reading link tag
		if(strcmp(it->tagName().c_str(), "a") == 0 && strcmp(it->closingText().c_str(), "</a>") == 0) {
			inLinkTag = true;
		}

		// Skip tags and comments
		if ((!it->isTag()) && (!it->isComment())) {

			// Trim extra blank space
			htmlcxx::HTML::single_blank(it->text());

			// Skip html entities
			htmlcxx::HTML::decode_entities(it->text());

			// Test encode
			if(!htmlcxx::HTML::detect_utf8(it->text().c_str(), it->text().size()) ) {
				Encoder::decode_html_entities_utf8((char *)it->text().c_str(), NULL);
			}

			// Tokanize
			tk = std::strtok((char *)it->text().c_str(), TOKEN_DELIMITERS);

			while(tk != NULL) {
				// Convert term to lower case
				Encoder::toLower(tk);

				// Add term to the vocabulary
				std::string term(tk);
				if(term.size() > 0) {
					termId = lexon->contains(term);
					if(termId == NOT_IN_VOCABULARY) {
						termId = lexon->insert(term);
						lexon->vocabularyInsert(termId);
					}
					run->insert(docId, termId);

					// Add anchor text
					if(inLinkTag) {
						anchorText->insertTerm(termId, docId);
					}
				}

				tk = std::strtok(NULL, TOKEN_DELIMITERS);
			}

			inLinkTag = false;
		}
	}

	// Print links
	if(docLinks.size() > 0) {
		(*docLinksFile) << docId << " " << docLinks.size();
		for(docLinksItr = docLinks.begin(); docLinksItr!=docLinks.end(); docLinksItr++) {
			(*docLinksFile) << " " << (*docLinksItr);
		}
		(*docLinksFile) << "\n";
	}

	doc.clear();
}

std::string Parser::preprocessQueryTerm(std::string str) {
	htmlcxx::HTML::decode_entities(str);
	char *tk = std::strtok((char *)str.c_str(), TOKEN_DELIMITERS);
	Encoder::toLower(tk);
	return std::string(tk);
}



